# -*- coding: utf-8 -*-
# @Time    : 2019/10/23 20:13
# @Author  : zhouqiang
# @Site    : 
# @File    : index_fitting.py
# @Software: PyCharm
# -*- coding: utf-8 -*-

"""
todo
asset_pool和server模块中有些回归的地方用的是这个脚本中的process函数
该脚本本来在factor_database中，现在项目管理，需要分离factor_database和asset_pool这些模块
暂时把这个脚本复制一份放在这里，之后需要改为regression_tools里面的函数
"""

# scipy.optimize.minimize包回归时，会自动加多进程
# 下面语句将其多进程关掉，使用我们自己的多进程
# 保证每个CPU最多100%占用
# 但这两句要放在import numpy之前，不然会失效
# 如果需要此设置，每次都将其放在代码最顶层
import copy
import os

os.environ['OMP_NUM_THREADS'] = '1'

import pandas as pd
import numpy as np
import statsmodels.api as sm
from quant_researcher.quant.project_tool.logger.my_logger import LOG
from quant_researcher.quant.datasource_fetch.fund_api import fund_nav_related, fund_info
from quant_researcher.quant.datasource_fetch.factor_api import factor_return_related
from quant_researcher.quant.datasource_fetch.common_data_api import t_trade_date
from quant_researcher.quant.datasource_fetch.index_api import index_price_related
from quant_researcher.quant.project_tool import assert_tool, code_processor
from quant_researcher.quant.project_tool.wrapper_tools import common_wrappers, conn_wrappers
from quant_researcher.quant.project_tool import time_tool
from quant_researcher.quant.project_tool.math_func import math_tool
from quant_researcher.quant.project_tool import hammer
from quant_researcher.quant.project_tool import df_tool
from quant_researcher.quant.project_tool.db_operator import db_conn, my_mysql
from quant_researcher.quant.project_tool.common_var import PERIOD_DICT
from quant_researcher.quant.datasource_fetch.index_api.index_constant import ZZ_CODE_LIST, SW_CODE_LIST
from scipy.optimize import minimize

C_DATE = 'tradedate'
CODE_CLASSES = ['index', 'fund', 'manager']
OTHER_CONFIG = ['industry', 'style', 'asset']
FACTORS_IN_F_C = ['bond_factors', 'fama']
IMPLEMENTED_FACTORS = ['bond_factors', 'fama', 'barra']
IMPLEMENTED_CONFIG = ['industry', 'style', 'asset',
                      'bond_factors', 'fama', 'barra', 'sector']
T_FUND_ASSET_INDEX_FITTING = 'mf_di_fndassetconfig'
T_MGR_ASSET_INDEX_FITTING = 'mf_di_mgrassetconfig'
T_INDEX_INDUSTRY_INDEX_FITTING = 'mf_di_idxindustryconfig'
T_FUND_INDUSTRY_INDEX_FITTING = 'mf_di_fndindustryconfig_test'
T_MGR_INDUSTRY_INDEX_FITTING = 'mf_di_mgrindustryconfig_test'
T_INDEX_STYLE_INDEX_FITTING = 'mf_di_idxstyleconfig'
T_FUND_STYLE_INDEX_FITTING = 'mf_di_fndstyleconfig_test'
T_MGR_STYLE_INDEX_FITTING = 'mf_di_mgrstyleconfig_test'
T_INDEX_FACTOR_INDEX_FITTING = 'mf_di_idxfactorconfig'
T_INDEX_FACTOR_INDEX_FITTING_BARRA = 'mf_di_idxfactorconfig_barra'
T_FUND_FACTOR_INDEX_FITTING = 'mf_di_fndfactorconfig'
T_MGR_FACTOR_INDEX_FITTING = 'mf_di_mgrfactorconfig'
T_FUND_FACTOR_INDEX_FITTING_BARRA = 'mf_di_fndfactorconfig_barra'
T_MGR_FACTOR_INDEX_FITTING_BARRA = 'mf_di_mgrfactorconfig_barra'
INDEX_NEEDED_4_STYLE_JC = [
    '399372', '399373', '399374', '399375', '399376', '399377'
]  # 巨潮风格指数
INDEX_NEEDED_4_STYLE_SZ1000 = ['399630', '399631']  # 深证1000
INDEX_NEEDED_4_STYLE_GZ = ['399370', '399371']  # 国证风格
INDEX_NEEDED_4_ASSET_FIVE_ASSETS = [
    '000300',  # 沪深300
    'HSI',  # 恒生指数
    'CBA00301.CS',  # 中债总财富（总值）指数
    'AUCI'  # 黄金价格
]


def get_cumulative(a_lst, sign):
    """
    累加 或 累乘

    :param a_lst: 数据
    :param sign: 符号，加还是乘
    :return: list
    """
    tmp = 0 if sign == '+' else 1
    ans = []
    for x in a_lst:
        if sign == '+':
            tmp += x
        if sign == '*':
            tmp *= x
        ans.append(tmp)
    return ans


def fit_indexes_4_a_code(u_code, code_class, start, end, **kwargs):
    index_hq_df = kwargs.pop('index_hq_df')
    x_cols = kwargs.pop('x_cols')
    i_or_s_specific = kwargs.pop('i_or_s_specific')
    i_or_s = kwargs.pop('i_or_s')
    c_c = f'{code_class}_code'
    i_name = kwargs.pop('i_name')
    tmp_conn = kwargs.pop('conn_dict')['derivative_data']

    fund_type = None
    if code_class == 'manager':
        # 2019年12月25日14:48:36，为什么这个写在这里面，基金的又是在外面呢
        # 因为不是同一阶段开发的，再去修改外面的就不太方便了
        u_code, fund_type = u_code

    step_using = 40
    if i_or_s in ['fama', 'barra']:
        config_col_prefix = 'factor'
    elif i_or_s in OTHER_CONFIG:
        config_col_prefix = i_or_s
    elif i_or_s == 'bond_factors':
        config_col_prefix = 'factor'
        # 2019年12月25日09:22:25，将债券因子的拟合区间缩短成了三个月，这儿要对应修改
        step_using = 30
    elif i_or_s == 'sector':
        config_col_prefix = 'sector'
    else:
        raise NotImplementedError(i_or_s)

    if code_class == 'fund':
        df = fund_nav_related.get_fund_return(u_code, start, end, conn=tmp_conn)
    elif code_class == 'manager':
        df = fund_nav_related.get_manager_return(u_code, fund_type, start, end,conn=tmp_conn)
    else:
        df = index_price_related.get_index_return(u_code, start, end,conn=tmp_conn)

    if (df is None) or (df.shape[0] < step_using):
        LOG.warning(f'{code_class}，'
                    f'{u_code} 的行情记录条数不满足要求，'
                    f'which is：条数>={step_using}')
        return

    y_col = f'y_{code_class}_{u_code}'
    df = df.rename(columns={'end_date': 'tj', 'daily_return': y_col})
    if code_class == 'fund':
        del df['fund_code']
    elif code_class == 'index':
        del df['index_code']
    elif code_class == 'manager':
        df = df.drop(columns=['manager_code', 'fund_type'])

    else:
        raise NotImplementedError

    tj_in_index_hq_df = index_hq_df['tj'].values[0]
    tj_in_fund_hq_df = df['tj'].values[0]
    LOG.info(f'基金{u_code}行情中日期列示例：{tj_in_fund_hq_df}，'
             f'class：{tj_in_fund_hq_df.__class__}。'
             f'指数行情中日期列示例：{tj_in_index_hq_df}，'
             f'class：{tj_in_index_hq_df.__class__}。')
    assert tj_in_index_hq_df.__class__ == tj_in_fund_hq_df.__class__

    df = df_tool.merge([df, index_hq_df], how='inner', key='tj', drop_na=True)
    df.index = range(df.shape[0])
    hammer.slim(
        df, name=f'{u_code} 用到的HQ', deal_float=False, show_index=1, a_f=1
    )
    if df.shape[0] == 0:
        LOG.warning(f'{code_class} {u_code} 的行情与指数行情inner join之后为空，'
                    '说明两个DF的时间区间没有任何重叠，一定是遇到了严重的行情问题，'
                    'code行情的开始和结束（查询条件，用这个开始和结束去数据库查查）：'
                    f'{start}，{end}；'
                    f'指数行情的开始和结束(实际)：{index_hq_df["tj"].values[0]}，'
                    f'{index_hq_df["tj"].values[-1]}，但是这也不一定就是错误，'
                    '所以直接返回，而不抛出异常')
        return

    data = []
    end_date = end

    LOG.info(f'将要存储的end_date={end_date}')

    # hammer.slim(x_s, name='X')
    # hammer.slim(y_s, name='Y')
    if i_or_s_specific == 'BARRA_FYZ':
        x_s = df[x_cols]
        y_s = df[y_col]
        x_s = sm.add_constant(x_s)
        obj_func = lambda b: ((x_s.dot(b) - y_s) ** 2).sum()
        bnds = np.array(list((-np.inf, np.inf) for _ in range(x_s.shape[1])))
        bnds[11:-1] = [0, 1]
        bnds[-1] = [1, 1]
        cons = [{'type': 'eq', 'fun': lambda x: np.sum(x[11:-1]) - 1}]
        x0 = np.zeros(x_s.shape[1])
        fit_ans = minimize(obj_func, x0=x0, bounds=bnds, constraints=cons)
    else:
        x_s = df_tool.df_to_np_array(df[x_cols])
        y_s = df_tool.df_to_np_array(df[[y_col]])
        if i_or_s in ['barra', 'industry', 'style', 'asset', 'sector']:
            fit_ans = math_tool.fit_with_constraints_by_array(
                x_s=x_s, y_s=y_s, with_intercept=True, cpt_r_square=True,
                no_bounds=True, set_constraints=False
            )
        else:
            fit_ans = math_tool.fit_with_constraints_by_array(
                x_s=x_s, y_s=y_s, with_intercept=True, cpt_r_square=True,
            )

    if i_or_s_specific != 'BARRA_FYZ':
        if fit_ans is None:
            LOG.warning(f'优化未完成，参数：u_code={u_code}，'
                        f'code_class={code_class}，start={start}，end={end}')
            return

        w_ans = [float(x) for x in fit_ans['weight'][0]]

        r2 = float(fit_ans['r_square'])

        alpha = float(fit_ans['intercept'])
    else:
        if not fit_ans['success']:
            LOG.warning(f'优化未完成，参数：u_code={u_code}，'
                        f'code_class={code_class}，start={start}，end={end}')
            return
        w_ans = [float(x) for x in fit_ans.x[1:]]
        alpha = float(fit_ans.x[0])
        specret = y_s - x_s.dot(fit_ans.x)
        sse = (specret ** 2).sum()
        sst = ((y_s - y_s.mean()) ** 2).sum()
        r2 = float(1 - (sse / sst))

    if not -100 <= r2 <= 100:
        LOG.error(f'{u_code}, {code_class}, {start}, {end}, got inf r2')
        r2 = None

    if not -100 <= alpha <= 100:
        LOG.error(f'{u_code}, {code_class}, {start}, {end}, got inf alpha')
        alpha = None

    for idx, a_beta in enumerate(w_ans):
        an_index = x_cols[idx].replace('index_', '', 1)
        index_rtn = get_cumulative(
            [float(x) + 1 for x in df[x_cols[idx]].values], sign='*'
        )[-1]
        index_rtn -= 1
        row = {
            c_c: u_code,
            'end_date': end_date,
            f'{config_col_prefix}_type': i_or_s_specific,
            f'{config_col_prefix}_code': an_index,
            'alpha': alpha,
            'r2': r2,
        }
        row.update({
            f'{config_col_prefix}_beta': a_beta,
            f'{config_col_prefix}_contribution': a_beta * index_rtn,
        })
        if code_class == 'manager':
            row.update({'fund_type': fund_type})
        # hammer.slim(row, name='行结果示例', a_f=1)
        data.append(row)
    if i_or_s in ['style', 'sector', 'industry']:
        total_ret = (y_s + 1).prod() - 1
        data_df = pd.DataFrame(data)
        data_df['period_interval'] = PERIOD_DICT[i_name][0]
        data_df = data_df.rename(columns={f"{config_col_prefix}_contribution": f"{config_col_prefix}_ret_contribution"})
        data_df['resid_ret_contribution'] = total_ret-data_df[f'{config_col_prefix}_ret_contribution'].sum()
        data_df[f'{config_col_prefix}_ret_contribution_pct'] = data_df[f'{config_col_prefix}_ret_contribution'] / total_ret
        data_df['resid_ret_contribution_pct'] = data_df['resid_ret_contribution'] / total_ret
        # 风险贡献计算
        # 参考Barra文档
        # Risk Contribution is Exposure times
        # Volatility times Correlation
        # Decomposing risk using the x-sigma-rho formula
        x_s_df = pd.DataFrame(x_s)
        # x_s_cov = x_s_df.cov()
        sigma_p = np.std(y_s, ddof=1)
        factor_vol = x_s_df.std()
        factor_p_corr = x_s_df.corrwith(pd.Series(y_s.flatten()))
        weight_array = data_df[f'{config_col_prefix}_beta']
        trc = weight_array * factor_vol * factor_p_corr
        spec_trc = sigma_p - trc.sum()

        # weight_array = (data_df[f'{config_col_prefix}_beta'] / data_df[f'{config_col_prefix}_beta'].sum()).values.reshape(-1,1)
        # trc, sigma_p = factor_risk_contribution(weight_array, x_s_cov, specret=specret)
        # sigma_p = np.dot(x_s_cov,weight_array) / mrc
        data_df[f'{config_col_prefix}_risk_contribution'] = trc
        data_df[f'{config_col_prefix}_risk_contribution_pct'] = trc / sigma_p
        data_df['resid_risk_contribution'] = spec_trc
        data_df['resid_risk_contribution_pct'] = spec_trc / sigma_p
        data = data_df.to_dict(orient='records')
        # hammer.slim(pd.DataFrame(data), name=f'{code_class}({u_code})结果', a_f=0)

    return data


def for_a_code(code_info, **kwargs):
    if kwargs.get('code_class') == 'manager':
        return fit_indexes_4_a_code(code_info, **kwargs)
    elif kwargs.get('code_class') == 'index':
        return fit_indexes_4_a_code(code_info, **kwargs)
    else:
        return fit_indexes_4_a_code(code_info[0], **kwargs)


def get_index_hq(index_needed_lst, i_or_s, **kwargs):
    start = kwargs.pop('start')
    end = kwargs.pop('end')
    bond_factors_hq = kwargs.pop('bond_factors_hq')
    fama_factors_hq = kwargs.pop('fama_factors_hq')
    barra_factors_hq = kwargs.pop('barra_factors_hq')

    index_hq_df = None
    for an_index in index_needed_lst:
        an_index_hq_df = index_price_related.get_index_return(an_index, start, end)
        if an_index_hq_df is None:
            LOG.warning(f'指数 {an_index}，从 {start} 到 {end} 没有行情')
            return
        an_index_hq_df = an_index_hq_df.drop(columns=['index_code'])
        an_index_hq_df = an_index_hq_df.rename(columns={'end_date': 'tj', 'daily_return': f'index_{an_index}'})
        tj_in_index_hq_df = an_index_hq_df['tj'].values[0]
        LOG.warning(f'用到的指数{an_index}行情日期，示例：{tj_in_index_hq_df}，'
                    f'class：{tj_in_index_hq_df.__class__}。')
        if index_hq_df is None:
            index_hq_df = an_index_hq_df
        else:
            index_hq_df = df_tool.merge(
                [index_hq_df, an_index_hq_df], how='inner', key='tj'
            )
    if i_or_s == 'bond_factors':
        index_hq_df = bond_factors_hq
    if i_or_s == 'fama':
        index_hq_df = fama_factors_hq
    if i_or_s == 'barra':
        index_hq_df = barra_factors_hq
    hammer.slim(index_hq_df, show_index=1, name='用到的指数行情', a_f=1)

    x_cols = list(index_hq_df.columns)
    x_cols.remove('tj')
    LOG.info('用到的指数：%s', '，'.join(x_cols))
    # assert False

    return index_hq_df, x_cols


def check_parameters(i_or_s, i_or_s_specific, code_class):
    if code_class != 'others':
        assert_tool.value_in_list(code_class, CODE_CLASSES)
    assert_tool.value_in_list(i_or_s, the_list=IMPLEMENTED_CONFIG)
    table = None
    prefix = i_or_s
    if i_or_s == 'industry':
        assert_tool.value_in_list(i_or_s_specific, ['ZZ10', 'SW28'])
        if code_class == 'fund':
            table = T_FUND_INDUSTRY_INDEX_FITTING
        elif code_class == 'manager':
            table = T_MGR_INDUSTRY_INDEX_FITTING
        else:
            table = T_INDEX_INDUSTRY_INDEX_FITTING
    elif i_or_s == 'style':
        assert_tool.value_in_list(i_or_s_specific, ['JC', 'GZ', 'SZ1000'])
        if code_class == 'fund':
            table = T_FUND_STYLE_INDEX_FITTING
            # table = 'mf_di_fndstyleconfig_test'
        elif code_class == 'manager':
            table = T_MGR_STYLE_INDEX_FITTING
            # table = 'mf_di_mgrstyleconfig_test'
        else:
            table = T_INDEX_STYLE_INDEX_FITTING
    elif i_or_s == 'asset':
        assert_tool.value_in_list(i_or_s_specific, ['FIVE_ASSETS'])
        if code_class == 'fund':
            table = T_FUND_ASSET_INDEX_FITTING
        elif code_class == 'manager':
            table = T_MGR_ASSET_INDEX_FITTING
        else:
            raise NotImplementedError
    elif i_or_s == 'bond_factors':
        assert_tool.value_in_list(i_or_s_specific, ['BOND_4YZ', 'BOND_7YZ'])
    elif i_or_s == 'fama':
        assert_tool.value_in_list(i_or_s_specific, ['FAMA_3YZ', 'FAMA_5YZ'])
    elif i_or_s == 'barra':
        assert_tool.value_in_list(i_or_s_specific, ['BARRA_SYZ', 'BARRA_FYZ'])
        prefix = 'factor'
        if code_class == 'fund':
            table = T_FUND_FACTOR_INDEX_FITTING_BARRA
        elif code_class == 'manager':
            table = T_MGR_FACTOR_INDEX_FITTING_BARRA
        else:
            table = T_INDEX_FACTOR_INDEX_FITTING_BARRA
    elif i_or_s == 'sector':
        assert_tool.value_in_list(i_or_s_specific, ['SW_SECTOR'])
        if code_class == 'fund':
            table = 'mf_di_fndsectorconfig_test'
        elif code_class == 'manager':
            table = 'mf_di_mgrsectorconfig_test'
    else:
        raise NotImplementedError(i_or_s)

    if i_or_s in FACTORS_IN_F_C:
        prefix = 'factor'
        if code_class == 'fund':
            table = T_FUND_FACTOR_INDEX_FITTING
        elif code_class == 'manager':
            table = T_MGR_FACTOR_INDEX_FITTING
        else:
            table = T_INDEX_FACTOR_INDEX_FITTING
    if i_or_s in ['industry', 'style', 'sector']:
        p_key = [
            f'{code_class}_code', 'end_date', f'{prefix}_code', f'{prefix}_type', 'period_interval'
        ]
    else:
        p_key = [
            f'{code_class}_code', 'end_date', f'{prefix}_code', f'{prefix}_type'
        ]
    if code_class == 'manager':
        p_key.insert(1, 'fund_type')
    return table, p_key


def get_start_and_end(today, d_or_m, i_name):
    start, end = time_tool.find_interval_start(
        end_date=today, interval_name=i_name, freq=d_or_m
    )
    return start, end


@conn_wrappers.conn_meddler(conn_getter=db_conn.get_derivative_data_conn)
def get_distinct_fund_benchmark(**kwargs):
    conn = kwargs.pop('conn')
    extra_filter = kwargs.pop('extra_filter', None)
    if extra_filter is None:
        tmp_df = my_mysql.read_v2(select='max(end_date) as max_tj', sfrom='mf_di_fndbenchmark', conn=conn)
        end_date = tmp_df['max_tj'].values[0]
        extra_filter = f'end_date=\'{end_date}\''
    select = f'distinct benchmark'
    df = my_mysql.read_v2(
        select=select, sfrom='mf_di_fndbenchmark', where=extra_filter, conn=conn
    )
    ans = list(df['benchmark'].values)
    return ans


def find_indexes_needed_fitting():
    return get_distinct_fund_benchmark()


def get_finder_and_kwds(code_class, target_fund_type_1):
    if code_class == 'fund':
        finder = fund_info.get_fund_benchmark
        kwds_4_finder = {
            'select': ['fund_code', 'benchmark'],
            'fund_type_1': target_fund_type_1,
            'convert_2_list': True
        }
    elif code_class == 'manager':
        finder = fund_info.get_manager_benchmark
        kwds_4_finder = {
            'fund_type': target_fund_type_1,
            'select': [fund_info.M_C, fund_info.C_F_T],
            'convert_2_list': True
        }
    else:
        finder = find_indexes_needed_fitting
        kwds_4_finder = {}

    return finder, kwds_4_finder


def get_index_needed(i_or_s, i_or_s_specific, **kwargs):
    append_currency_index = kwargs.pop('append_currency_index', True)
    if i_or_s == 'industry':
        if i_or_s_specific == 'ZZ10':
            index_needed_lst = copy.deepcopy(ZZ_CODE_LIST)
        else:
            index_needed_lst = copy.deepcopy(SW_CODE_LIST)
    elif i_or_s == 'bond_factors':
        index_needed_lst = []
    elif i_or_s in ['fama', 'barra']:
        index_needed_lst = []
    elif i_or_s == 'asset':
        index_needed_lst = copy.deepcopy(INDEX_NEEDED_4_ASSET_FIVE_ASSETS)
    elif i_or_s == 'style':
        if i_or_s_specific == 'JC':
            index_needed_lst = copy.deepcopy(INDEX_NEEDED_4_STYLE_JC)
        elif i_or_s_specific == 'GZ':
            index_needed_lst = copy.deepcopy(INDEX_NEEDED_4_STYLE_GZ)
        elif i_or_s_specific == 'SZ1000':
            index_needed_lst = copy.deepcopy(INDEX_NEEDED_4_STYLE_SZ1000)
        else:
            raise NotImplementedError('目前只有：GZ，JC，SZ1000')
    elif i_or_s == 'sector':
        if i_or_s_specific == 'SW_SECTOR':
            index_needed_lst = ['TKCONSUME', 'TKCYCLE','TKFINANCE','TKTMT','TKMANUFACTURE','TKOTHER']
        else:
            raise NotImplementedError('目前只有：SW_SECTOR')
    else:
        raise NotImplementedError('目前只有：债券，fama，行业，风格，资产, 板块')
    if append_currency_index:
        # 不管是 中证行业 还是 申万行业，还是五花八门的风格指数 都加上货币指数
        index_needed_lst.append('H11025')
    LOG.info('找到的指数：%s', '，'.join(index_needed_lst))
    return index_needed_lst


@common_wrappers.count_time
def process(**kwargs):
    test_codes = kwargs.pop('test_codes', None)
    test_count = kwargs.pop('test_count', None)
    process_number = kwargs.pop('process_number', None)
    i_or_s = kwargs.pop('i_or_s')
    i_or_s_specific = kwargs.pop('i_or_s_specific')
    today = kwargs.pop('today')
    code_class = kwargs.pop('code_class')
    data_4_api = kwargs.pop('data_4_api', False)
    i_name = kwargs.pop('i_name', None)

    table, p_key = check_parameters(
        i_or_s, i_or_s_specific, code_class=code_class
    )
    if i_name is None:
        i_name = '3m' if i_or_s == 'bond_factors' else '1y'
    start, end = get_start_and_end(today, 'D', i_name=i_name)
    if start is None:
        start = '1900-01-01'
    LOG.info(f'结果表={table}，主键={p_key}，{i_name}，开始={start}，结束={end}')

    bond_factors_hq = None
    fama_factors_hq = None
    barra_factors_hq = None
    if i_or_s == 'bond_factors':
        if code_class == 'fund':
            target_fund_type_1 = ['20', '30']
        elif code_class == 'manager':
            target_fund_type_1 = ['00', '20', '30']
        elif code_class == 'index':
            target_fund_type_1 = None
        else:
            raise NotImplementedError
        bond_factors_hq = factor_return_related.get_bond_factor_return(
            i_or_s_specific, start, end
        )
        bond_factors_hq = bond_factors_hq.rename(columns={'end_date': 'tj'})
        # hammer.slim(bond_factors_hq)
    elif i_or_s in ['fama', 'barra']:
        if code_class == 'fund':
            target_fund_type_1 = ['10', '20']
        elif code_class == 'manager':
            target_fund_type_1 = ['00', '10', '20']
        elif code_class == 'index':
            target_fund_type_1 = None
        else:
            raise NotImplementedError
        if i_or_s == 'fama':
            fama_factors_hq = factor_return_related.get_fama_factor_return(
                i_or_s_specific, start, end
            )
            fama_factors_hq = fama_factors_hq.rename(columns={'end_date': 'tj'})
            # hammer.slim(fama_factors_hq)
        if i_or_s == 'barra':
            barra_factors_hq = factor_return_related.get_barra_factor_return(
                i_or_s_specific, start, end
            )
            barra_factors_hq = barra_factors_hq.rename(columns={'end_date': 'tj'})
            # hammer.slim(barra_factors_hq)
    elif i_or_s == 'asset':
        if code_class == 'fund':
            target_fund_type_1 = '20'
        elif code_class == 'manager':
            target_fund_type_1 = ['00', '20']
        elif code_class == 'index':
            target_fund_type_1 = None
        else:
            raise NotImplementedError
    elif i_or_s == 'sector':
        if code_class == 'fund':
            target_fund_type_1 = ['10', '20']
        elif code_class == 'manager':
            target_fund_type_1 = ['00', '10', '20']
        elif code_class == 'index':
            target_fund_type_1 = None
        else:
            raise NotImplementedError
    else:
        if code_class == 'fund':
            target_fund_type_1 = ['10', '20']
        elif code_class == 'manager':
            target_fund_type_1 = ['00', '10', '20']
        elif code_class == 'index':
            target_fund_type_1 = None
        else:
            raise NotImplementedError

    append_currency_index = False
    # if i_or_s in ['style', 'industry']:
    #     append_currency_index = False
    index_needed_lst = get_index_needed(i_or_s, i_or_s_specific,
                                        append_currency_index=append_currency_index)
    index_hq_df, x_cols = get_index_hq(
        index_needed_lst, i_or_s,
        start=start, end=end,
        bond_factors_hq=bond_factors_hq,
        fama_factors_hq=fama_factors_hq,
        barra_factors_hq=barra_factors_hq,
    )
    kwds_4_code_processor = {
        'code_class': code_class,
        'start': start, 'end': end,
        'i_or_s': i_or_s, 'i_or_s_specific': i_or_s_specific,
        'index_hq_df': index_hq_df, 'x_cols': x_cols, 'i_name': i_name
    }

    finder, kwds_4_finder = get_finder_and_kwds(
        code_class, target_fund_type_1
    )

    if not data_4_api:
        cpu = code_processor.CodeProcessorV1(
            func_to_find=finder,
            func_to_deal=for_a_code,
            # func_to_save=None,  # 如果注释了，则采用此工具类默认的结果处理函数
            kwds_to_find=kwds_4_finder,
            kwds_to_deal=kwds_4_code_processor,
            kwds_to_save={
                'table': table, 'p_key': p_key, 'db': 'derivative_data_write',
                'replace_nan': True
            },
            prepare_these_conn=['basic_data', 'derivative_data', 'derivative_data_write'],
            handler_frequency='code',
            test_codes=test_codes, test_count=test_count,
            process_number=process_number,
            data_4_api=data_4_api,
        )
        cpu.run()
    else:
        cpu = code_processor.CodeProcessorV1(
            func_to_find=finder,
            func_to_deal=for_a_code,
            # func_to_save=None,  # 如果注释了，则采用此工具类默认的结果处理函数
            kwds_to_find=kwds_4_finder,
            kwds_to_deal=kwds_4_code_processor,
            prepare_these_conn=['basic_data', 'derivative_data'],
            handler_frequency='code',
            test_codes=test_codes, test_count=test_count,
            process_number=process_number,
            data_4_api=data_4_api,
        )
        api_res = cpu.run()
        df_2_concat_list = []
        for f_res in api_res:
            f_df = pd.DataFrame(f_res)
            df_2_concat_list.append(f_df)
        res_df = pd.concat(df_2_concat_list, sort=False, ignore_index=True)
        return res_df


def process_all_kinds_2_core(today=None, code_class_lst=None, i_or_s_lst=None):
    if today is None:
        today = time_tool.get_today()
        today = time_tool.get_the_end_of_last_month(today)
    i_or_s_needed = OTHER_CONFIG if i_or_s_lst is None else i_or_s_lst
    code_class_needed = CODE_CLASSES if code_class_lst is None else code_class_lst
    for code_class in code_class_needed:
        for i_or_s in i_or_s_needed:
            if i_or_s == 'style':
                i_or_s_specific_s = ['JC']
            elif i_or_s == 'industry':
                i_or_s_specific_s = ['ZZ10', 'SW28']
            elif i_or_s == 'asset':
                if code_class == 'index':
                    continue  # 指数没有这个
                i_or_s_specific_s = ['FIVE_ASSETS']
            elif i_or_s == 'sector':
                i_or_s_specific_s = ['SW_SECTOR']
            else:
                raise NotImplementedError(i_or_s)
            for i_or_s_specific in i_or_s_specific_s:
                for period in ['1y','6m','3m','incep']:
                    process(
                        code_class=code_class,
                        today=today,
                        i_or_s=i_or_s,
                        i_or_s_specific=i_or_s_specific,
                        i_name=period
                    )


def process_all_kinds_1_core(today=None, code_class_lst=None, i_or_s_lst=None):
    if today is None:
        today = time_tool.get_yesterday()
    if not t_trade_date.is_a_trade_date(the_day=today):
        return
    i_or_s_needed = IMPLEMENTED_FACTORS if i_or_s_lst is None else i_or_s_lst
    code_class_needed = CODE_CLASSES if code_class_lst is None else code_class_lst
    for code_class in code_class_needed:
        for i_or_s in i_or_s_needed:
            if i_or_s == 'bond_factors':
                i_or_s_specific_s = ['BOND_4YZ', 'BOND_7YZ']
            elif i_or_s == 'fama':
                i_or_s_specific_s = ['FAMA_3YZ', 'FAMA_5YZ']
            elif i_or_s == 'barra':
                i_or_s_specific_s = ['BARRA_SYZ', 'BARRA_FYZ']
            else:
                raise NotImplementedError(i_or_s)
            for i_or_s_specific in i_or_s_specific_s:
                process(
                    code_class=code_class,
                    today=today,
                    i_or_s=i_or_s,
                    i_or_s_specific=i_or_s_specific
                )


@common_wrappers.get_interval_from_env
def initialize_1(date1=None, date2=None, **kwargs):
    date1 = '2017-01-01' if date1 is None else date1
    date2 = time_tool.get_yesterday() if date2 is None else date2

    # 因子配置：债券，第一指数，barra，fama
    time_tool.run_from_start_to_end(
        func=process_all_kinds_1_core, start_date=date1, end_date=date2, step='daily',
        kwds_4_func=kwargs
    )


@common_wrappers.get_interval_from_env
def initialize_2(date1=None, date2=None):
    date1 = '2017-01-01' if date1 is None else date1
    date2 = time_tool.get_yesterday() if date2 is None else date2

    # 行业，资产，风格配置
    time_tool.run_from_start_to_end(
        func=process_all_kinds_2_core, start_date=date1, end_date=date2, step='monthly'
    )


@common_wrappers.automatic_initialization_checker(func_2_initialize=initialize_1)
def process_all_kinds_1(today=None, code_class_lst=None, i_or_s_lst=None):
    process_all_kinds_1_core(today=today, code_class_lst=code_class_lst, i_or_s_lst=i_or_s_lst)


@common_wrappers.automatic_initialization_checker(func_2_initialize=initialize_2)
def process_all_kinds_2(today=None, code_class_lst=None, i_or_s_lst=None):
    process_all_kinds_2_core(today=today, code_class_lst=code_class_lst, i_or_s_lst=i_or_s_lst)


@common_wrappers.count_time
def calc_history(start, end, code_class_lst, i_or_s_lst):
    today = end
    while today >= start:
        process_all_kinds_1(today=today, code_class_lst=code_class_lst,
                            i_or_s_lst=i_or_s_lst)
        today = time_tool.date_shifter(
            before=today, step='days', how_many=-1, fmt_str='YYYY-MM-DD'
        )


@common_wrappers.count_time
def calc_history_2(start, end, code_class_lst, i_or_s_lst):
    today = end
    while today >= start:
        process_all_kinds_2(today=today, code_class_lst=code_class_lst,
                            i_or_s_lst=i_or_s_lst)
        today = time_tool.date_shifter(
            before=today, step='quarters', how_many=-1, fmt_str='YYYY-MM-DD'
        )


if __name__ == '__main__':
    # process_all_kinds_1(code_class_lst=['fund', 'manager'],
    #                     i_or_s_lst=['barra', 'bond_factors', 'fama'])

    process_all_kinds_2(code_class_lst=['fund', 'manager'],
                        i_or_s_lst=['industry', 'style', 'sector'])
    # calc_history_2('2014-12-31','2020-11-30',
    #                code_class_lst=['fund','manager'],i_or_s_lst=['sector','style','industry'])
    # calc_history_2('2014-12-31','2020-09-30',
    #                code_class_lst=['fund'],i_or_s_lst=['style'])
    # process(
    #     code_class='fund',
    #     today='2020-09-30',
    #     i_or_s='style',
    #     i_or_s_specific='JC',
    #     i_name='6m',
    #     test_codes=[('110022', '000300')],
    # )
